/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package sinaEx;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import weibo4j.Friendships;
import weibo4j.Timeline;
import weibo4j.Users;
import weibo4j.Weibo;
import weibo4j.model.Paging;
import weibo4j.model.Status;
import weibo4j.model.User;
import weibo4j.model.WeiboException;

/**
 *
 * @author Administrator
 */
public final class CelebrityDataCollector {

    public static int time = 0;
    public static List<String> userid = new ArrayList<String>();
    public static Map<String, String> username = new HashMap<String, String>();
    public static Map<String, String> ctag = new HashMap<String, String>();
    public static Map<String, Boolean> verified = new HashMap<String, Boolean>();
    public static Map<String, Boolean> fame = new HashMap<String, Boolean>();
    public static Map<String, Integer> friendsCount = new HashMap<String, Integer>();
    public static Map<String, Integer> fansCount = new HashMap<String, Integer>();
    public static Map<String, Double> blogFrequency = new HashMap<String, Double>();
    public static Map<String, Map<String, Integer>> interestTag = new HashMap<String, Map<String, Integer>>();
    public static Map<String, Integer> mediaCount = new HashMap<String, Integer>();
    public static Map<String, Integer> agencyCount = new HashMap<String, Integer>();
    public static Map<String, Integer> celebrityCount = new HashMap<String, Integer>();
    public static Map<String, Integer> normalCount = new HashMap<String, Integer>();
    public static Map<String, Integer> websiteCount = new HashMap<String, Integer>();
    public static Map<String, Integer> brandCount = new HashMap<String, Integer>();
    public static Map<String, Integer> campusCount = new HashMap<String, Integer>();
    public static Map<String, Integer> zoombieCount = new HashMap<String, Integer>();
    public static Map<String, Integer> zoombieBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> campusBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> brandBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> websiteBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> normalBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> celebrityBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> agencyBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> mediaBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> createBlogCount = new HashMap<String, Integer>();
    public static Map<String, String[]> celebrityType = new HashMap<String, String[]>();
    public static List<String> media = new ArrayList<String>();
    public static List<String> brand = new ArrayList<String>();
    public static List<String> agency = new ArrayList<String>();
    public static List<String> website = new ArrayList<String>();
    public static List<String> campus = new ArrayList<String>();
    public static List<String> celebrity = new ArrayList<String>();
    public static List<String> zoombie = new ArrayList<String>();
    public static List<String> pickup = new ArrayList<String>();
    public static int statusC = 0;

    public static void main(String[] args) throws IOException, WeiboException, SQLException, InterruptedException {
        prepareWeibo();
        CelebrityDataCollector data = new CelebrityDataCollector();
    }

    public static void prepareWeibo() {
        String accessToken = spider.Auth4Code.getCode();
        Weibo weibo = new Weibo();
        weibo.setToken(accessToken);
    }

    CelebrityDataCollector() throws IOException, WeiboException, InterruptedException {
        readCelebrity();
        readZoombie();
        int i = 0;
        //String c="1220924217";
        for (String c : celebrity) {
            
            if (i < 3121) {
            i += 1;
            continue;
            }
             
            System.out.println((i++) + " / " + celebrity.size() + " " + c);
            Users um = new Users();
            User userinfo = null;
            try {
                userinfo = um.showUserById(c);
                time += 1;
            } catch (Exception e) {
                if (e.getMessage().contains("User does not exists!")) {
                    System.out.println(c + " does not exist.");
                    continue;
                }
                int flag = 0;
                int retry=0;
                while (userinfo == null) {
                    retry+=1;
                    if(retry>15)break;
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    if (time > 10000) {
                        Thread.sleep(300000);
                        System.out.println("sleeping");
                    }
                    try {
                        userinfo = um.showUserById(c);
                    } catch (Exception ee) {
                        if (ee.getMessage().contains("User does not exists!")) {
                            System.out.println(c + " does not exist.");
                            break;
                        }
                    }
                    if (userinfo != null) {
                        time = 1;
                    } else {
                        break;
                    }
                }
            }

            time += 1;
            if (userinfo == null) {
                continue;
            }
            statusC = userinfo.getStatusesCount();
            getFriendsCountTag(c);
            getBlogFrequency(c);
            writeData(c);
            clear();
        }
        /*
        String uid = "1751885472";
        userid.add(uid);
        getFriendsCountTag(uid);
        getBlogFrequency(uid);
        writeData(uid);
        clear();
        int size = userid.size();
        int i = 1;
        System.out.println("Time: " + i + ". Current size is: " + (size));
        while (size < 50000) {
        List<String> pickuptemp = new ArrayList<String>();
        pickuptemp.addAll(pickup);
        pickup.clear();
        for (String id : pickuptemp) {
        if (userid.contains(id)) {
        continue;
        }
        getFriendsCountTag(id);
        getBlogFrequency(id);
        writeData(id);
        clear();
        size = userid.size();
        System.out.println("Time: " + i + ". Current size is: " + (size) + " " + id);
        i += 1;
        }
        }
         * 
         */
    }

    public void readZoombie() throws IOException {
        String filename = "src/sinaEx/zoombie";
        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
        String line = reader.readLine();
        while (line != null) {
            if (!line.isEmpty()) {
                String s[] = line.split(" ");
                zoombie.add(s[0]);
            }
            line = reader.readLine();
        }
        reader.close();
    }

    public void readCelebrity() throws IOException {
        String filename = "src/sinaEx/ex_empty_BF";
        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
        String line = reader.readLine();
        //line = reader.readLine();
        while (line != null) {
            /*
            String s[] = line.split("\t");
            String id = s[1];
            String type[] = {s[2], s[3], s[4]};
            celebrityType.put(s[1], type);
            if (s[4].equals("2")) {
            media.add(id);
            }
            if (s[4].equals("1")) {
            celebrity.add(id);
            }
            if (s[4].equals("3")) {
            brand.add(id);
            }
            if (s[4].equals("4") || s[4].equals("7")) {
            agency.add(id);
            }
            if (s[4].equals("5")) {
            website.add(id);
            }
            if (s[4].equals("6")) {
            campus.add(id);
            }
            celebrityType.put(id, type);
             *
             */
            celebrity.add(line);
            line = reader.readLine();
        }
        reader.close();
    }

    //Get Tag of a celebrity
    public void getCelebrityTag(String uid, String id) {
        Map<String, Integer> tagCount = new HashMap<String, Integer>();
        int count = 0;
        if (interestTag.containsKey(uid)) {
            tagCount = interestTag.get(uid);
        }
        String key = celebrityType.get(id)[1];
        if (tagCount.containsKey(key)) {
            count = tagCount.get(key);
        }
        count += 1;
        tagCount.put(key, count);
        interestTag.put(uid, tagCount);
    }

    //Get a user;
    public void selectNextUsers(String uid) {
    }

    //Get all the information of a user;
    public void getFriendsCountTag(String uid) throws WeiboException, InterruptedException {

        int friendscount = basicInfo(uid);
        getCount(uid, friendscount);
    }

    //Basic info like uid, verified, friends count, fans count and username;
    public int basicInfo(String uid) throws WeiboException, InterruptedException {

        Users um = new Users();
        User userinfo = null;
        try {
            userinfo = um.showUserById(uid);
            time += 1;
        } catch (Exception e) {
            int flag = 0;
            while (userinfo == null) {
                if (flag == 0) {
                    System.out.println("time out!");
                }
                flag = 1;
                if (time > 10000) {
                    //Thread.sleep(300000);
                    //System.out.println("sleeping");
                }
                try {
                    userinfo = um.showUserById(uid);
                } catch (Exception ee) {
                }

                if (userinfo != null) {
                    time = 1;
                }
            }
        }
        time += 1;
        boolean v = userinfo.isVerified();
        if (celebrity.contains(uid)) {
            fame.put(uid, true);
        } else {
            fame.put(uid, false);
        }
        int friendscount = userinfo.getFriendsCount();
        int fanscount = userinfo.getFollowersCount();
        String uname = userinfo.getName();
        username.put(uid, uname);
        verified.put(uid, v);
        friendsCount.put(uid, friendscount);
        fansCount.put(uid, fanscount);
        return friendscount;
    }

    //All kinds of count;
    public void getCount(String uid, int friendscount) throws WeiboException, InterruptedException {
        Friendships fm = new Friendships();
        List<User> users = new ArrayList<User>();
        int page = (int) Math.ceil((double) friendscount / (double) 200);
        for (int i = 0; i < page; i++) {
            List<User> temp = null;
            try {
                temp = fm.getFriendsByID(uid, 200, i * 200);
                time += 1;
            } catch (Exception e) {
                int flag = 0;
                while (temp == null) {
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    if (time > 10000) {
                        //Thread.sleep(300000);
                        //System.out.println("sleeping");
                    }
                    try {
                        temp = fm.getFriendsByID(uid, 200, i * 200);
                    } catch (Exception ee) {
                    }
                    if (temp != null) {
                        time = 1;
                    }
                }
            }

            users.addAll(temp);
        }
        int mediacount = 0;
        int agencycount = 0;
        int zoombiecount = 0;
        int campuscount = 0;
        int websitecount = 0;
        int brandcount = 0;
        int celebritycount = 0;
        int normalcount = 0;
        for (User u : users) {
            String id = u.getId();
            int flag = 0;
            if (media.contains(id)) {
                flag = 1;
                mediacount += 1;
                mediaCount.put(uid, mediacount);
            }
            if (agency.contains(id)) {
                flag = 1;
                agencycount += 1;
                agencyCount.put(uid, agencycount);
            }
            if (campus.contains(id)) {
                flag = 1;
                campuscount += 1;
                campusCount.put(uid, campuscount);
            }
            if (website.contains(id)) {
                flag = 1;
                websitecount += 1;
                websiteCount.put(uid, websitecount);
            }
            if (brand.contains(id)) {
                flag = 1;
                brandcount += 1;
                brandCount.put(uid, brandcount);
            }
            if (zoombie.contains(id)) {
                flag = 1;
                zoombiecount += 1;
                zoombieCount.put(uid, zoombiecount);
            }
            if (celebrity.contains(id)) {
                flag = 1;
                celebritycount += 1;
                //getCelebrityTag(uid, id);
                celebrityCount.put(uid, celebritycount);
            }
            if (flag == 0) {
                normalcount += 1;
                normalCount.put(uid, normalcount);
            }
        }
    }

    public void getBlogFrequency(String uid) throws WeiboException, InterruptedException {
        Timeline tm = new Timeline();
        List<Status> status = null;
        try {
            status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
            int retry=0;
            while (status == null) {
                try {
                    if(retry++>10)break;
                    status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
                } catch (Exception ee) {
                }
            }
            int retry2=0;
            while (status.isEmpty() && statusC > 10) {
                if(retry2++>10)break;
                int flag = 0;
                if (time > 10000) {
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    //Thread.sleep(300000);
                    //System.out.println("sleeping");
                }
                try {
                    status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
                } catch (Exception ee) {
                }
                if (status != null) {
                    time = 1;
                }
            }
            time += 1;
        } catch (Exception e) {
            int retry=0;
            while (status == null) {
                try {
                    if(retry++>10)break;
                    status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
                } catch (Exception ee) {
                }
            }
            int retry2=0;
            while (status.isEmpty() && statusC != 0) {
                if(retry2++>10)break;
                int flag = 0;
                if (time > 10000) {
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    //Thread.sleep(300000);
                    //System.out.println("sleeping");
                }
                try {
                    status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);

                } catch (Exception ee) {
                }
                if (status != null) {
                    time = 1;
                }
            }
        }
        time += 1;
        int sSize = status.size();
        if (sSize == 0) {
            System.out.println("Status size is 0???");
        }
        if (sSize > 1) {
            long latestTime = status.get(0).getCreatedAt().getTime();
            long oldestTime = status.get(sSize - 1).getCreatedAt().getTime();
            double microblogFrequency = (double) sSize / ((latestTime - oldestTime) / 1000 / 60 / 60 / 24);
            blogFrequency.put(uid, microblogFrequency);
        } else {
            double microblogFrequency = 0;
            blogFrequency.put(uid, microblogFrequency);
        }


        int createcount = 0;
        int mediacount = 0;
        int agencycount = 0;
        int campuscount = 0;
        int brandcount = 0;
        int websitecount = 0;
        int celebritycount = 0;
        int normalcount = 0;
        int zoombiecount = 0;
        for (Status s : status) {
            if (s.getRetweetedStatus() != null) {
                Status os = s.getRetweetedStatus();
                if (os.getUser() != null) {
                    String id = os.getUser().getId();
                    if (media.contains(id)) {
                        mediacount += 1;
                        mediaBlogCount.put(uid, mediacount);
                    } else if (agency.contains(id)) {
                        agencycount += 1;
                        agencyBlogCount.put(uid, agencycount);
                    } else if (campus.contains(id)) {
                        campuscount += 1;
                        campusBlogCount.put(uid, campuscount);
                    } else if (website.contains(id)) {
                        websitecount += 1;
                        websiteBlogCount.put(uid, websitecount);
                    } else if (brand.contains(id)) {
                        brandcount += 1;
                        brandBlogCount.put(uid, brandcount);
                    } else if (zoombie.contains(id)) {
                        zoombiecount += 1;
                        zoombieBlogCount.put(uid, zoombiecount);
                    } else if (celebrity.contains(id)) {
                        celebritycount += 1;
                        celebrityBlogCount.put(uid, celebritycount);
                    } else {
                        if (!id.equals(uid)) {
                            normalcount += 1;
                            normalBlogCount.put(uid, normalcount);
                        } else {
                            createcount += 1;
                            createBlogCount.put(uid, createcount);
                        }
                    }
                }
            } else {
                createcount += 1;
                createBlogCount.put(uid, createcount);
            }
        }
    }

    public void writeData(String id) {
        if (userid.contains(id)) {
            System.out.println("Repeated.");
            return;
        }
        userid.add(id);
        System.out.println("Written");
        /*
        String interest = "";
        Map<String, Integer> tagCount = interestTag.get(id);
        if (tagCount != null) {
        Set keys = tagCount.keySet();
        int count = keys.size();
        for (int i = 0; i < count; i++) {
        String key = (String) keys.toArray()[i];
        int c = tagCount.get(key);
        interest += (key + "|" + c + "||");
        }
        }
         * 

        String query = "INSERT INTO `weibonews`.`wb_ex_celebrity` (`uid`, `username`, `friendsCount`, `fansCount`, `fame`,`verified`,`tag`, `interestTag`, `mediaCount`, `agencyCount`, `celebrityCount`, `normalCount`, `websiteCount`, `brandCount`, `campusCount`,`zoombieCount`,`blogFrequency`,`agencyRC`, `mediaRC`, `zoombieRC`, `normalRC`, `campusRC`,  `websiteRC`, `celebrityRC`, `create`) VALUES ("
        + id + ","
        + "'" + username.get(id) + "',"
        + friendsCount.get(id) + ","
        + fansCount.get(id) + ","
        + fame.get(id) + ","
        + verified.get(id) + ","
        + "'" + celebrityType.get(id)[1] + "',"
        + "'" + interest + "'" + ","
        + mediaCount.get(id) + ","
        + agencyCount.get(id) + ","
        + celebrityCount.get(id) + ","
        + normalCount.get(id) + ","
        + websiteCount.get(id) + ","
        + brandCount.get(id) + ","
        + campusCount.get(id) + ","
        + zoombieCount.get(id) + ","
        + blogFrequency.get(id) + ","
        + agencyBlogCount.get(id) + ","
        + mediaBlogCount.get(id) + ","
        + zoombieBlogCount.get(id) + ","
        + normalBlogCount.get(id) + ","
        + campusBlogCount.get(id) + ","
        + websiteBlogCount.get(id) + ","
        + celebrityBlogCount.get(id) + ","
        + createBlogCount.get(id)
        + ");";
        


        try {

        Class.forName("com.mysql.jdbc.Driver").newInstance();
        Connection conn = (Connection) DriverManager.getConnection("jdbc:mysql://192.168.8.6:3306/weibonews?useUnicode=true&characterEncoding=gb2312", "root", "root");
        Statement stmt = (Statement) conn.createStatement();
        ResultSet result = (ResultSet) stmt.executeQuery("select * from `weibonews`.`wb_ex_celebrity` where `uid` = '" + id + "';");
        if (result.next()) {
        System.out.println(result.first());
        System.out.println("returned");
        return;
        }
        // stmt.executeUpdate(delete);
        stmt.executeUpdate(query);
        conn.close();
         *
         */
        if (!agencyBlogCount.containsKey(id)) {
            agencyBlogCount.put(id, 0);
        }
        if (!mediaBlogCount.containsKey(id)) {
            mediaBlogCount.put(id, 0);
        }

        if (!createBlogCount.containsKey(id)) {
            createBlogCount.put(id, 0);
        }
        if (!celebrityBlogCount.containsKey(id)) {
            celebrityBlogCount.put(id, 0);
        }
        if (!websiteBlogCount.containsKey(id)) {
            websiteBlogCount.put(id, 0);
        }
        if (!campusBlogCount.containsKey(id)) {
            campusBlogCount.put(id, 0);
        }
        if (!normalBlogCount.containsKey(id)) {
            normalBlogCount.put(id, 0);
        }
        if (!zoombieBlogCount.containsKey(id)) {
            zoombieBlogCount.put(id, 0);
        }


        try {
            Class.forName("com.mysql.jdbc.Driver").newInstance();
            Connection conn = (Connection) DriverManager.getConnection("jdbc:mysql://192.168.8.6:3306/weibonews?useUnicode=true&characterEncoding=gb2312", "root", "root");
            Statement stmt = (Statement) conn.createStatement();
            String query = "update `weibonews`.`wb_ex_celebrity` set `blogFrequency`=" + blogFrequency.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `agencyRC`=" + agencyBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `mediaRC`=" + mediaBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `zoombieRC`=" + zoombieBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `normalRC`=" + normalBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `campusRC`=" + campusBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `websiteRC`=" + websiteBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `celebrityRC`=" + celebrityBlogCount.get(id) + " where `uid`='" + id + "';";
            stmt.executeUpdate(query);
            query = "update `weibonews`.`wb_ex_celebrity` set `create`=" + createBlogCount.get(id) + " where `uid`='" + id + "';";

            // stmt.executeUpdate(delete);
            stmt.executeUpdate(query);
            conn.close();
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("Already Exist!");
        }
    }

    private void clear() {
        agencyBlogCount.clear();
        agencyCount.clear();
        blogFrequency.clear();
        brandBlogCount.clear();
        brandCount.clear();
        campusBlogCount.clear();
        campusCount.clear();
        celebrityBlogCount.clear();
        celebrityCount.clear();
        createBlogCount.clear();
        fansCount.clear();
        friendsCount.clear();
        interestTag.clear();
        mediaBlogCount.clear();
        mediaCount.clear();
        normalBlogCount.clear();
        normalCount.clear();
        username.clear();
        verified.clear();
        websiteBlogCount.clear();
        websiteCount.clear();
        zoombieBlogCount.clear();
        zoombieCount.clear();
    }
}
